﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Text.RegularExpressions;
using mshtml;
using System.Collections;
using Informaiton_Retrieval.com.ir.Model;
using PorterStemmerAlgorithm;
using Informaiton_Retrieval.com.ir.utli;

namespace Informaiton_Retrieval.com.ir.Model
{
    class VeryLargeFile
    {
        const int MaxLines = 10000;
        //const string ID = "WARC-Target-URI: ";
        //const string ID = "WARC-Record-ID:";
        const string ID = "WARC-TREC-ID:";
        MainModel model;
        long _fileLines;
        public IndexDictionary _indexInfo = new IndexDictionary();
        //List<string> tempFilePaths = new List<string>();

        public VeryLargeFile(MainModel model)
        {
            this.model = model;
        }

        public long CalculateLines(string filePath)
        {
            using (StreamReader sr = new StreamReader(filePath, Encoding.UTF8))
            {
                model.LoadedLine("Calculating File,please wait...");
                _fileLines = 0;
                while (sr.ReadLine() != null)
                {
                    _fileLines++;
                }
                return _fileLines;
            }
        }

        public void LoadData(string filePath)
        {
            int p = 0;

            string webId = "";
            bool warcStartFlag = false, htmlStartFlag = false;

            StringBuilder eachHtml = new StringBuilder();
            Dictionary<string, StringBuilder> totalHtmlD = new Dictionary<string, StringBuilder>();

            using (StreamReader sr = new StreamReader(filePath, Encoding.UTF8))
            {
                string line;
                long currentLines = 0;
                while ((line = sr.ReadLine()) != null)
                {
                    currentLines++;
                    culatePercentLine(currentLines);
                    if (line.Contains("WARC/"))
                        warcStartFlag = true;
                    if (warcStartFlag)
                    {
                        if (line.Contains(ID))
                        {
                            webId = line.Replace(ID, "").Trim();
                            Console.WriteLine(webId);
                        }
                        if (line.Contains("Content-Length:"))
                            warcStartFlag = false;
                    }

                    if (!warcStartFlag)
                    {
                        if (line.Contains("<html"))
                            htmlStartFlag = true;
                        if (htmlStartFlag)

                            eachHtml.AppendLine(line);

                        if (line.Contains("</html>"))
                        {
                            htmlStartFlag = false;
                            totalHtmlD.Add(webId, eachHtml);
                            eachHtml = new StringBuilder();
                            if (totalHtmlD.Count == 5)
                            {
                                p++;
                                saveDictionaryTemp(p.ToString(), totalHtmlD);
                                totalHtmlD = new Dictionary<string, StringBuilder>();
                                Console.WriteLine("part:" + p);
                            }
                        }
                    }
                }
                if (totalHtmlD.Count == 0) { }
                else
                {
                    p++;
                    saveDictionaryTemp(p.ToString(), totalHtmlD);
                }
            }
            //model.OnFileLoadComplete();
        }

        private void culatePercentLine(long line)
        {
            long lp = (line * 100) / _fileLines;
            int i = (int)lp;
            setPercent(i);
        }

        private void culatePercentIndex(int key)
        {
            long lp = (key * 100) / _indexInfo.Keys.Count;
            int i = (int)lp;
            setPercent(i);
        }

        private void setPercent(int i)
        {
            model.LoadFileProgress(i);
            model.LoadedLine(i + "%");
        }

        public void SaveOutput(string path)
        {
            setPercent(0);
            saveDictionary(path);
            saveIndex(path);
        }

        private void saveIndex(string path)
        {
            string outputPath = path;
            string filename = "index";
            string newPath = System.IO.Path.Combine(outputPath, "output");
            System.IO.Directory.CreateDirectory(newPath);
            TextWriter tw = new StreamWriter(@newPath + "/" + filename + ".txt", false, Encoding.UTF8);

            int keyCount = 0;

            foreach (string key in _indexInfo.Keys)
            {
                keyCount++;
                culatePercentIndex(keyCount);
                tw.Write("" + key + "," + _indexInfo[key].counter + ":");
                foreach (string key1 in _indexInfo[key]._dicPos.Keys)
                {
                    string tempInt = "";
                    for (int i = 0; i < _indexInfo[key]._dicPos[key1].Count; i++)
                    {
                        if (_indexInfo[key]._dicPos[key1].Count - 1 == i)
                        {
                            tempInt = tempInt + _indexInfo[key]._dicPos[key1][i];
                        }
                        else
                        {
                            tempInt = tempInt + _indexInfo[key]._dicPos[key1][i] + ",";
                        }
                    }
                    tw.Write("<" + key1 + "," + _indexInfo[key]._dicPos[key1].Count + ":<" + tempInt + ">;");
                }
                tw.WriteLine(">");
            }
            tw.Close();
        }

        private void saveDictionary(string path)
        {

            string outputPath = path;
            string filename = "dictionary";
            string newPath = System.IO.Path.Combine(outputPath, "output");
            System.IO.Directory.CreateDirectory(newPath);
            TextWriter tw = new StreamWriter(@newPath + "/" + filename + ".txt", false, Encoding.UTF8);

            foreach (string key in _indexInfo.Keys)
            {
                tw.WriteLine("<" + key + ":" + _indexInfo[key].counter + ">");
            }

            tw.Close();

        }

        private void saveDictionaryTemp(string filename, Dictionary<string, StringBuilder> totalHtml)
        {
            string outputPath = "";
            string newPath = System.IO.Path.Combine(outputPath, "temp");
            System.IO.Directory.CreateDirectory(newPath);
            string tempPath = @newPath + "/" + "data" + filename + ".dat";

            foreach (string key in totalHtml.Keys)
            {
                StringBuilder sb = totalHtml[key];
                string[] ary = regexString(sb.ToString());
                makeIndex(key, ary);
            }
        }

        private void makeIndex(string key, string[] ary)
        {
            for (int i = 0; i < ary.Length; i++)
            {
                DocumentData indexstr = new DocumentData();
                List<int> tempIntList = new List<int>();
                if (_indexInfo.ContainsKey(ary[i]))
                {
                    _indexInfo[ary[i]].counter++;
                    if (_indexInfo[ary[i]]._dicPos.ContainsKey(key))
                    {
                        tempIntList = _indexInfo[ary[i]]._dicPos[key];
                        tempIntList.Add(i);
                        _indexInfo[ary[i]]._dicPos[key] = tempIntList;
                    }
                    else
                    {
                        tempIntList.Add(i);
                        _indexInfo[ary[i]]._dicPos.Add(key, tempIntList);
                    }
                }
                else
                {
                    indexstr.counter = 1;
                    tempIntList.Add(i);
                    indexstr._dicPos.Add(key, tempIntList);
                    _indexInfo.Add(ary[i], indexstr);
                }
            }
        }

        private string[] regexString(string str)
        {
            str = Regex.Replace(str, @"<script[\d\D]*?>[\d\D]*?</script>", String.Empty);
            str = Regex.Replace(str, @"<[^>]*>", String.Empty);

            //char[] delimiter1 = new char[] { ',', '\n', '\t', ' ', '\r', '|', ';' ,'(' , ')' , '^','[',']'};
            string[] strArray = Regex.Split(str, @"\W+|_");
            //string[] strArray = str.Split(delimiter1, StringSplitOptions.RemoveEmptyEntries);

            char[] del = new char[] { ';', '[', ']', '(', ')', '{', '}', '.', ':', '"', '^', '/', '\'', '_' };
            for (int i = 0; i < strArray.Length; i++)
                strArray[i] = strArray[i].Trim(del);

            if (model.pre.Stopword)
                strArray = Tokenization.StopWord(strArray);//
            if (model.pre.CaseFold)
                strArray = Tokenization.LowerCase(strArray);//
            if (model.pre.Stemming)
                strArray = Tokenization.Stemming(strArray);//
            strArray = Tokenization.FiltNumber(strArray);
            strArray = Tokenization.DelEmpty(strArray);
            //strArray = FiltSame(strArray);
            //strArray = SortArray(strArray);

            return strArray;
        }

        //        private string[] FiltNumber(string[] input)
        //        {
        //            List<string> strList = new List<string>();
        //            foreach (string s in input)
        //            {

        //                if (!IsNumber(s))
        //                    strList.Add(s);
        //            }
        //            return strList.ToArray();
        //        }

        //        private string[] FiltSame(string[] input)
        //        {
        //            List<string> strList = new List<string>();
        //            foreach (string s in input)
        //                if (!strList.Contains(s))
        //                    strList.Add(s);
        //            return strList.ToArray();
        //        }

        //        private string[] LowerCase(string[] input)
        //        {
        //            List<string> strList = new List<string>();
        //            foreach (string s in input)
        //            {
        //                string ls = s.ToLower();

        //                strList.Add(ls);
        //            }
        //            return strList.ToArray();
        //        }

        //        private string[] StopWord(string[] input)
        //        {
        //            List<string> strList = new List<string>();
        //            foreach (string s in input)
        //            {
        //                string ls = s;
        //                ls = Regex.Replace(s,stopwords, String.Empty);

        //                strList.Add(ls);
        //            }
        //            return strList.ToArray();
        //        }

        //        private string[] DelEmpty(string[] input)
        //        {
        //            List<string> strList = new List<string>();
        //            foreach (string s in input)
        //            {
        //                if (!String.IsNullOrEmpty(s))
        //                    strList.Add(s);
        //            }
        //            return strList.ToArray();
        //        }

        //        private string[] SortArray(string[] input)
        //        {
        //            List<string> strList = new List<string>();
        //            foreach (string s in input)
        //                strList.Add(s);
        //            strList.Sort();
        //            return strList.ToArray();
        //        }

        //        private string[] Stemming(string[] input) {
        //            StemmerInterface sf = new PorterStemmer();
        //            List<string> strList = new List<string>();
        //            foreach (string s in input)
        //            {
        //                string ss = sf.stemTerm(s);
        //                strList.Add(ss);
        //            }
        //            return strList.ToArray();
        //        }


        //        bool IsNumber(string text)
        //        {
        //            Regex regex = new Regex(@"^[-+]?[0-9]*\.?[0-9]+$");
        //            return regex.IsMatch(text);
        //        }

        //        const string stopwords = @"\b(?:a|an|the|and|but|if|or|because|as|until|while|of|at|by|for|with|about|against|between|into|through|
        //                                            during|before|after|above|below|to|from|up|down|in|out|on|off|over|under|again|further|then|once|
        //                                            here|there|when|where|why|how|all|any|both|each|few|more|most|other|some|such|no|nor|not|only|
        //                                            so|than|too|very|one|every|least|less|many|now|ever|never|also|just|whether|like|well|back|even|
        //                                            still|way|since|another|however|let's|that's|who's|what's|here's|there's|when's|where's|
        //                                            why's|how's|won't|wouldn't|shan't|shouldn't|can't|cannot|couldn't|mustn't|isn't|aren't|wasn't|
        //                                            weren't|hasn't|haven't|hadn't|doesn't|don't|didn't|i'm|you're|he's|she's|it's|we're|they're|i've|
        //                                            you've|we've|they've|i'd|you'd|he'd|she'd|we'd|they'd|i'll|you'll|he'll|she'll|we'll|they'll|me|my|
        //                                            myself|we|us|our|ours|ourself|you|your|yours|yourself|yourselves|he|him|his|hiself|she|her|hers|
        //                                            herself|it|its|itself|they|them|their|theirs|themselves|what|which|who|whom|this|that|these|those|
        //                                            am|is|are|was|were|be|been|being|have|has|had|having|do|does|did|doing|will|would|should|could|ought|
        //                                            may|might|must|shall)\b";

    }
}
